{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "dcbc5d62",
   "metadata": {},
   "source": [
    "![数据示例](https://www.runoob.com/wp-content/uploads/2021/06/6A6DE9DA-E0EE-4001-8C21-1D6A8EBF70FF.jpeg)\n",
    "\n",
    "空值\n",
    "- n/a\n",
    "- NA\n",
    "- —\n",
    "- na"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6c81d5a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://static.runoob.com/download/property-data.csv\n",
    "# 下载到本地\n",
    "test_file='/Users/msxr/develop/tmp/property-data.csv'\n",
    "import pandas as pd\n",
    "df = pd.read_csv(test_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fe685236",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0      3\n",
      "1      3\n",
      "2    NaN\n",
      "3      1\n",
      "4      3\n",
      "5    NaN\n",
      "6      2\n",
      "7      1\n",
      "8     na\n",
      "Name: NUM_BEDROOMS, dtype: object\n"
     ]
    }
   ],
   "source": [
    "print (df['NUM_BEDROOMS'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "95cf6673",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    False\n",
      "1    False\n",
      "2     True\n",
      "3    False\n",
      "4    False\n",
      "5     True\n",
      "6    False\n",
      "7    False\n",
      "8    False\n",
      "Name: NUM_BEDROOMS, dtype: bool\n"
     ]
    }
   ],
   "source": [
    "print (df['NUM_BEDROOMS'].isnull())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d6be4687",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           PID  ST_NUM     ST_NAME OWN_OCCUPIED NUM_BEDROOMS NUM_BATH SQ_FT\n",
      "0  100001000.0   104.0      PUTNAM            Y            3        1  1000\n",
      "1  100002000.0   197.0   LEXINGTON            N            3      1.5    --\n",
      "2  100003000.0     NaN   LEXINGTON            N          NaN        1   850\n",
      "3  100004000.0   201.0    BERKELEY           12            1      NaN   700\n",
      "4          NaN   203.0    BERKELEY            Y            3        2  1600\n",
      "5  100006000.0   207.0    BERKELEY            Y          NaN        1   800\n",
      "6  100007000.0     NaN  WASHINGTON          NaN            2   HURLEY   950\n",
      "7  100008000.0   213.0     TREMONT            Y            1        1   NaN\n",
      "8  100009000.0   215.0     TREMONT            Y           na        2  1800\n",
      "--------\n",
      "           PID  ST_NUM    ST_NAME OWN_OCCUPIED NUM_BEDROOMS NUM_BATH SQ_FT\n",
      "0  100001000.0   104.0     PUTNAM            Y            3        1  1000\n",
      "1  100002000.0   197.0  LEXINGTON            N            3      1.5    --\n",
      "8  100009000.0   215.0    TREMONT            Y           na        2  1800\n"
     ]
    }
   ],
   "source": [
    "# 删除包含空数据的行\n",
    "new_df = df.dropna()  # 不修改df\n",
    "print(df)\n",
    "print('-'*8)\n",
    "print(new_df.to_string())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0ac99d4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(inplace = True) # 修改df\n",
    "df.dropna(subset=['ST_NUM'], inplace = True) # 修改指定列\n",
    "df.fillna(12345, inplace = True) # 空值替换为12345\n",
    "df['PID'].fillna(12345, inplace = True) # 指定列的空值替换为12345"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "336f41e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PID</th>\n",
       "      <th>ST_NUM</th>\n",
       "      <th>ST_NAME</th>\n",
       "      <th>OWN_OCCUPIED</th>\n",
       "      <th>NUM_BEDROOMS</th>\n",
       "      <th>NUM_BATH</th>\n",
       "      <th>SQ_FT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100001000.0</td>\n",
       "      <td>104.0</td>\n",
       "      <td>PUTNAM</td>\n",
       "      <td>Y</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100002000.0</td>\n",
       "      <td>197.0</td>\n",
       "      <td>LEXINGTON</td>\n",
       "      <td>N</td>\n",
       "      <td>3</td>\n",
       "      <td>1.5</td>\n",
       "      <td>--</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>100009000.0</td>\n",
       "      <td>215.0</td>\n",
       "      <td>TREMONT</td>\n",
       "      <td>Y</td>\n",
       "      <td>na</td>\n",
       "      <td>2</td>\n",
       "      <td>1800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           PID  ST_NUM    ST_NAME OWN_OCCUPIED NUM_BEDROOMS NUM_BATH SQ_FT\n",
       "0  100001000.0   104.0     PUTNAM            Y            3        1  1000\n",
       "1  100002000.0   197.0  LEXINGTON            N            3      1.5    --\n",
       "8  100009000.0   215.0    TREMONT            Y           na        2  1800"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c6403ad8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "172.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 平均值 mean\n",
    "df['ST_NUM'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b1bc1d06",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "197.0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#中位数\n",
    "df[\"ST_NUM\"].median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "366b3105",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    3\n",
       "dtype: object"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 众数\n",
    "df[\"NUM_BEDROOMS\"].mode()"
   ]
  },
  {
   "cell_type": "raw",
   "id": "2aeff971",
   "metadata": {},
   "source": [
    "清洗格式错误数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a9503f15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           Date  duration\n",
      "day1 2020-12-01        50\n",
      "day2 2020-12-02        40\n",
      "day3 2020-12-26        45\n"
     ]
    }
   ],
   "source": [
    "data = {\n",
    "  \"Date\": ['2020/12/01', '2020/12/02' , '20201226'],\n",
    "  \"duration\": [50, 40, 45]\n",
    "}\n",
    "\n",
    "df = pd.DataFrame(data, index = [\"day1\", \"day2\", \"day3\"])\n",
    "\n",
    "df['Date'] = pd.to_datetime(df['Date'])\n",
    "\n",
    "print(df.to_string())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "98a8f231",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     name  age\n",
      "0  Google   50\n",
      "1  Runoob   40\n",
      "2  Taobao   30\n"
     ]
    }
   ],
   "source": [
    "person = {\n",
    "  \"name\": ['Google', 'Runoob' , 'Taobao'],\n",
    "  \"age\": [50, 40, 12345]    # 12345 年龄数据是错误的\n",
    "}\n",
    "\n",
    "df = pd.DataFrame(person)\n",
    "\n",
    "\n",
    "df.loc[2, 'age'] = 30 # 修改数据\n",
    "\n",
    "print(df.to_string())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ad605fe8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     name  age\n",
      "0  Google   50\n",
      "1  Runoob   40\n"
     ]
    }
   ],
   "source": [
    "person = {\n",
    "  \"name\": ['Google', 'Runoob' , 'Taobao'],\n",
    "  \"age\": [50, 40, 12345]    # 12345 年龄数据是错误的\n",
    "}\n",
    "\n",
    "df = pd.DataFrame(person)\n",
    "\n",
    "for x in df.index:\n",
    "  if df.loc[x, \"age\"] > 120:\n",
    "    df.drop(x, inplace = True)\n",
    "\n",
    "print(df.to_string())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ac24ff3f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     name  age\n",
      "0  Google   50\n",
      "1  Runoob   40\n",
      "3  Runoob   45\n",
      "4  Taobao   23\n"
     ]
    }
   ],
   "source": [
    "# 删除重复数据，可以直接使用drop_duplicates() 方法\n",
    "persons = {\n",
    "  \"name\": ['Google', 'Runoob', 'Runoob','Runoob', 'Taobao'],\n",
    "  \"age\": [50, 40, 40, 45, 23]  \n",
    "}\n",
    "\n",
    "df = pd.DataFrame(persons)\n",
    "\n",
    "df.drop_duplicates(inplace = True)\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1fbd9db7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
